use strict;
use DBI;
use DBD::ODBC;
use POSIX;
use Time::HiRes qw(time);
use Storable qw(dclone);
use List::Util qw(shuffle);
use List::MoreUtils qw/ uniq /;
use List::MoreUtils 'pairwise';

my $flagLowDim = 0;
my $iter = 0;

require 'CDS_FilterData.pl';

# fraction of total training data used for training. It doesn't change the amount of test data
my $trainFrac_CDS = 1.0;
my $trainFrac_NTDB = 0.0;

# number of training data files generated using same alive/dead patient ratio as the original data.
# NOTE: if trainFrac = 1.0, then nTrainFiles should be 1
my $nTrainFiles = 5;

if ($trainFrac_CDS == 1.0 and $trainFrac_NTDB == 1.0) {
    $nTrainFiles = 1;
}

my ($CDS_NTDB_HD,$CDS_onlyNTDB_LD,$CDS_NTDB_OUTPUT_TEST);

open (KEYINFO, '> CDS_NTDB_Imputation_Keys.txt');
open (LOG, '> CDS_NTDB_Imputation_Log.txt');

my $BASE_HD_FILE_NAME = 'CDS_NTDB_Imputation_HD';
my $BASE_LD_FILE_NAME = 'CDS_onlyNTDB_Imputation_LD';

open ($CDS_NTDB_OUTPUT_TEST, '> CDS_NTDB_Output_Test.txt'); # Output (y) for NTDB Test file


# Set connection parameters
my $dsn = 'dbi:ODBC:CDSDriver';
my ($username, $password) = ('trauma', 'trauma');
print LOG "username = $username, password = $password\n";

# Connect to the database
my $cds = DBI -> connect($dsn  ,$username,$password);

# Preparing and executing sql query
my $sql = "EXEC sp_CDS_Imputation_ExtractRelevantColsFromAllTables14";
print LOG "SQL COMMAND EXECUTED: $sql\n";

my $sth = $cds -> prepare($sql);
$sth -> execute();

# Declare hashes
my (%hUID_TREATMNT, %hUID, %hAGE, %hUID_PARUSE, %hUID_CHORIENTA, %hUID_SEATPOS_ROW, %hUID_SEATPOS_COL, %hUID_EJECTION, %hUID_GLASGOW, %hUID_AISCODE, %hUID_MANCOLL, %hUID_CLASS1, %hUID_CLASS2, %hUID_OBJCONT, %hDVTOTAL, %hUID_DVEST, %hSPLIMIT);
my (%hDEATH, %hTREATMNT, %hPARUSE, %hCHORIENTA, %hSEATPOS_ROW, %hSEATPOS_COL, %hEJECTION, %hGLASGOW, %hAISCODE, %hMANCOLL, %hCLASS1, %hCLASS2, %hOBJCONT, %hDVEST);

#########################################
# $UID = PSU_CASEID_ACCYEAR_VEHNO_OCCNO #
#########################################

# Declare corresponding hash keys
my ($DEATH, $TREATMNT, $UID, $AGE, $PARUSE, $CHORIENT, $CHORIENTA, $SEATPOS, $SEATPOS_ROW, $SEATPOS_COL, $EJECTION, $GLASGOW, $AISCODE, $MANCOLL, $CLASS1, $CLASS2, $OBJCONT, $DVTOTAL, $DVEST, $SPLIMIT);

print LOG "iter\tUID,\tAGE\tPARUSE\tCHORIENTA\tSEATPOS_ROW\tSEATPOS_COL\tEJECTION\tGLASGOW\tAISCODE\tMANCOLL\tCLASS1\tCLASS2\tOBJCONT\tDVTOTAL\tDVEST\tSPLIMIT\n";
my $startTime = time();
my @aUID_CDS;
my %hDataType;

while(($UID, $AGE, $PARUSE, $CHORIENT, $SEATPOS, $EJECTION, $GLASGOW, $TREATMNT, $DEATH, $AISCODE, $MANCOLL, $CLASS1, $CLASS2, $OBJCONT, $DVTOTAL, $DVEST, $SPLIMIT) = $sth -> fetchrow_array())
{
    $aUID_CDS[$iter] = $UID;
    
    $iter++;

    $AISCODE  = trimIfEmpty($AISCODE);

    $DEATH = filterDeath($DEATH);
    $TREATMNT = filterTreatmnt($TREATMNT);
    $PARUSE = filterParuse($PARUSE);
    $CHORIENTA = filterChorientA($PARUSE,$CHORIENT);
    ($SEATPOS_ROW, $SEATPOS_COL) = filterSeatpos($SEATPOS);
    $EJECTION = filterEjection($EJECTION);
    $GLASGOW = filterGlasgow($GLASGOW);
    $AISCODE = filterAiscode($AISCODE);
    $MANCOLL = filterMancoll($MANCOLL);
    $CLASS1 = filterClass($CLASS1);
    $CLASS2 = filterClass($CLASS2);
    $OBJCONT = filterObjcont($OBJCONT);
    $DVTOTAL = filterDvtotal($DVTOTAL);
    $DVEST = filterDvest($DVEST);
    $SPLIMIT = filterSplimit($SPLIMIT);
    
#    print LOG "$iter\t$UID\t$AGE\t$PARUSE\t$CHORIENTA\t$SEATPOS_ROW\t$SEATPOS_COL\t$EJECTION\t$GLASGOW\t$AISCODE\t$MANCOLL\t$CLASS1\t$CLASS2\t$OBJCONT\t$DVTOTAL\t$DVEST\t$SPLIMIT\n";
    
    # Start filling hashes
    $hUID{$UID} = 1;
    $hDEATH{$UID} = $DEATH;
    $hUID_TREATMNT{$UID}{$TREATMNT} = 1;
    $hAGE{$UID} = $AGE;
    $hUID_PARUSE{$UID}{$PARUSE} = 1;
    $hUID_CHORIENTA{$UID}{$CHORIENTA} = 1;
    $hUID_SEATPOS_ROW{$UID}{$SEATPOS_ROW} = 1;
    $hUID_SEATPOS_COL{$UID}{$SEATPOS_COL} = 1;
    $hUID_EJECTION{$UID}{$EJECTION} = 1;
    $hUID_GLASGOW{$UID}{$GLASGOW} = 1;
    $hUID_AISCODE{$UID}{$AISCODE} = 1;
    $hUID_MANCOLL{$UID}{$MANCOLL} = 1;
    $hUID_CLASS1{$UID}{$CLASS1} = 1;
    $hUID_CLASS2{$UID}{$CLASS2} = 1;
    $hUID_OBJCONT{$UID}{$OBJCONT} = 1;
    $hDVTOTAL{$UID} = $DVTOTAL;
    $hUID_DVEST{$UID}{$DVEST} = 1;
    $hSPLIMIT{$UID} = $SPLIMIT;
    
    $hDataType{$UID} = 'CDS';
}

@aUID_CDS = sort(uniq(@aUID_CDS));

my $runTime = time() - $startTime;
print LOG "Processed $iter rows of SQL data.\n";
print LOG "Time taken = $runTime seconds.\n\n";

############################################### NTDB DATA ########################################################
# Set connection parameters
$dsn = 'dbi:ODBC:NTDBNEWDriver';
# Connect to the database
my $ntdb = DBI -> connect($dsn,$username,$password);

# Preparing and executing sql query
$sql = "EXEC sp_NTDBNEW_Imputation_ExtractRelevantColsFromCDS14";
print LOG "SQL COMMAND EXECUTED: $sql\n";

$sth = $ntdb-> prepare($sql);
$sth -> execute();

my ($INC_KEY, $AISPCODE_PREDOT, $DEMO_AGE, $DISCHARGE_HOSPDISP, $ECODE_ECODE, $ECODE_ECODE2, $VITALS_GCSEYE, $VITALS_GCSVERB, $VITALS_GCSMOT, $VITALS_GCSTOT, $ED_EDDISP, $ED_YOINJ, $OUTPUT);
my (@aUID_NTDB_TRAIN,@aUID_NTDB_TEST);

$iter = 0;
while(($INC_KEY, $AISPCODE_PREDOT, $DEMO_AGE, $DISCHARGE_HOSPDISP, $ECODE_ECODE, $ECODE_ECODE2, $VITALS_GCSEYE, $VITALS_GCSVERB, $VITALS_GCSMOT, $VITALS_GCSTOT, $ED_EDDISP, $ED_YOINJ) = $sth -> fetchrow_array())
{
    if($ED_YOINJ == 2009 or $ED_YOINJ == 2010) {
        push(@aUID_NTDB_TEST, $INC_KEY);
        $hDataType{$INC_KEY} = 'NTDB_TEST';
    } else {
        push(@aUID_NTDB_TRAIN, $INC_KEY);
        $hDataType{$INC_KEY} = 'NTDB_TRAIN';
    }
        
    $iter++;
    
    $INC_KEY = trim($INC_KEY);
    $AISPCODE_PREDOT = trim($AISPCODE_PREDOT);
    $DISCHARGE_HOSPDISP = trimIfEmpty($DISCHARGE_HOSPDISP);
    $ECODE_ECODE = trimIfEmpty($ECODE_ECODE);
    $ECODE_ECODE2 = trimIfEmpty($ECODE_ECODE2);
    $VITALS_GCSEYE = trimIfEmpty($VITALS_GCSEYE);
    $VITALS_GCSVERB = trimIfEmpty($VITALS_GCSVERB);
    $VITALS_GCSMOT = trimIfEmpty($VITALS_GCSMOT);
    $VITALS_GCSTOT = trimIfEmpty($VITALS_GCSTOT);
    $ED_EDDISP = trimIfEmpty($ED_EDDISP);
    
    ($VITALS_GCSEYE, $VITALS_GCSVERB, $VITALS_GCSMOT, $VITALS_GCSTOT) = filterGCS_NTDB_forCDS($VITALS_GCSEYE, $VITALS_GCSVERB, $VITALS_GCSMOT, $VITALS_GCSTOT);
    $OUTPUT = filterStatus_NTDB_forCDS($DISCHARGE_HOSPDISP, $ED_EDDISP);
    
#    print LOG "$iter: $INC_KEY, $OUTPUT, $AISPCODE_PREDOT, $DEMO_AGE, $DISCHARGE_HOSPDISP, $ECODE_ECODE, $ECODE_ECODE2, $VITALS_GCSTOT, $ED_EDDISP, $ED_YOINJ\n";
    
    $hUID{$INC_KEY} = 1;
    $hDEATH{$INC_KEY} = $OUTPUT;
    $hUID_AISCODE{$INC_KEY}{$AISPCODE_PREDOT} = 1;
    $hAGE{$INC_KEY} = $DEMO_AGE;    
    $hUID_GLASGOW{$INC_KEY}{$VITALS_GCSTOT} = 1;
    $hDVTOTAL{$INC_KEY} = "NULL";
    $hSPLIMIT{$INC_KEY} = "NULL";
}

@aUID_NTDB_TRAIN = sort(uniq(@aUID_NTDB_TRAIN));
@aUID_NTDB_TEST = sort(uniq(@aUID_NTDB_TEST));

foreach my $key (sort keys(%hUID)) {
    %hTREATMNT = assignHKeys(\%{$hUID_TREATMNT{$key}},\%hTREATMNT);
    %hPARUSE = assignHKeys(\%{$hUID_PARUSE{$key}},\%hPARUSE);
    %hCHORIENTA = assignHKeys(\%{$hUID_CHORIENTA{$key}},\%hCHORIENTA);    
    %hSEATPOS_ROW = assignHKeys(\%{$hUID_SEATPOS_ROW{$key}},\%hSEATPOS_ROW);
    %hSEATPOS_COL = assignHKeys(\%{$hUID_SEATPOS_COL{$key}},\%hSEATPOS_COL);
    %hEJECTION = assignHKeys(\%{$hUID_EJECTION{$key}},\%hEJECTION);
    %hGLASGOW = assignHKeys(\%{$hUID_GLASGOW{$key}},\%hGLASGOW);
    %hAISCODE = assignHKeys(\%{$hUID_AISCODE{$key}},\%hAISCODE);
    %hMANCOLL = assignHKeys(\%{$hUID_MANCOLL{$key}},\%hMANCOLL);
    %hCLASS1 = assignHKeys(\%{$hUID_CLASS1{$key}},\%hCLASS1);
    %hCLASS2 = assignHKeys(\%{$hUID_CLASS2{$key}},\%hCLASS2);
    %hOBJCONT = assignHKeys(\%{$hUID_OBJCONT{$key}},\%hOBJCONT);
    %hDVEST = assignHKeys(\%{$hUID_DVEST{$key}},\%hDVEST);
}

my (@aInds_CDS_NULL, @aInds_CDS_Dead, @aInds_NTDB_TRAIN_Dead, @aInds_NTDB_TEST_Dead, @aInds_CDS_Alive, @aInds_NTDB_TRAIN_Alive, @aInds_NTDB_TEST_Alive);

my $counter = 0;

foreach my $key (@aUID_CDS) {
    if($hDEATH{$key} eq "NULL") {
        push(@aInds_CDS_NULL, $counter);
    } elsif($hDEATH{$key} == 1) {
        push(@aInds_CDS_Dead, $counter);
    } else {
        push(@aInds_CDS_Alive, $counter);
    }   
    $counter++;
}
foreach my $key (@aUID_NTDB_TRAIN) {
    if($hDEATH{$key} == 1) {
        push(@aInds_NTDB_TRAIN_Dead, $counter);
    } else {
        push(@aInds_NTDB_TRAIN_Alive, $counter);
    }
    $counter++;
}
foreach my $key (@aUID_NTDB_TEST) {
    if($hDEATH{$key} == 1) {
        push(@aInds_NTDB_TEST_Dead, $counter);
    } else {
        push(@aInds_NTDB_TEST_Alive, $counter);
    }
    $counter++;
}

my @aUID = @aUID_CDS;
push(@aUID, @aUID_NTDB_TRAIN);
push(@aUID, @aUID_NTDB_TEST);

print LOG "\nNumber of UID keys:\n hUID = " .($#aUID_CDS + 1) . "(CDS) + " . ($#aUID_NTDB_TRAIN + 1) . "(NTDB Train) + " . ($#aUID_NTDB_TEST + 1) . "(NTDB Test) = " . ($#aUID + 1) . " = " . (keys %hUID) . "\n hTREATMNT = " . (keys %hUID_TREATMNT) . "\n hAGE = " . (keys %hAGE) . "\n hUID_PARUSE = " . (keys %hUID_PARUSE) . "\n hUID_CHORIENTA = " . (keys %hUID_CHORIENTA) . "\n hUID_SEATPOS_ROW = " . (keys %hUID_SEATPOS_ROW) . "\n hUID_SEATPOS_COL = " . (keys %hUID_SEATPOS_COL) . "\n hUID_EJECTION = " . (keys %hUID_EJECTION) . "\n hUID_GLASGOW = " . (keys %hUID_GLASGOW) . "\n hUID_AISCODE = " . (keys %hUID_AISCODE) . "\n hUID_MANCOLL = " . (keys %hUID_MANCOLL) . "\n hUID_CLASS1 = " . (keys %hUID_CLASS1) . "\n hUID_CLASS2 = " . (keys %hUID_CLASS2) . "\n hUID_OBJCONT = " . (keys %hUID_OBJCONT) . "\n hDVTOTAL = " . (keys %hDVTOTAL) . "\n hUID_DVEST = " . (keys %hUID_DVEST) . "\n hSPLIMIT = " . (keys %hSPLIMIT) . "\n";
print LOG "\nPercent dead patients:\n CDS:\t\t\t" . ($#aInds_CDS_Dead + 1)/($#aUID_CDS + 1)*100 . "\n NTDB Train:\t" . ($#aInds_NTDB_TRAIN_Dead + 1)/($#aUID_NTDB_TRAIN + 1)*100 . "\n NTDB Test:\t\t" . ($#aInds_NTDB_TEST_Dead + 1)/($#aUID_NTDB_TEST + 1)*100 . "\n Overall:\t\t" . (($#aInds_CDS_Dead + 1)+($#aInds_NTDB_TRAIN_Dead + 1)+($#aInds_NTDB_TEST_Dead + 1))/($#aUID + 1)*100 . "\n\n";

# printing some counts
print LOG "Counts:\n------------------------------------------\n";
print LOG "TREATMNT\n";
foreach my $i (sort keys %hTREATMNT) {
    print LOG "$i $hTREATMNT{$i}\n";
}
print LOG "\nPARUSE\n";
foreach my $i (sort keys %hPARUSE) {
    print LOG "$i $hPARUSE{$i}\n";
}
print LOG "\nCHORIENTA\n";
foreach my $i (sort keys %hCHORIENTA) {
    print LOG "$i $hCHORIENTA{$i}\n";
}
print LOG "\nSEATPOS_ROW\n";
foreach my $i (sort keys %hSEATPOS_ROW) {
    print LOG "$i $hSEATPOS_ROW{$i}\n";
}
print LOG "\nSEATPOS_COL\n";
foreach my $i (sort keys %hSEATPOS_COL) {
    print LOG "$i $hSEATPOS_COL{$i}\n";
}
print LOG "\nEJECTION\n";
foreach my $i (sort keys %hEJECTION) {
    print LOG "$i $hEJECTION{$i}\n";
}
print LOG "\nGLASGOW\n";
foreach my $i (sort keys %hGLASGOW) {
    print LOG "$i $hGLASGOW{$i}\n";
}
print LOG "\nAISCODE\n";
foreach my $i (sort keys %hAISCODE) {
    print LOG "$i $hAISCODE{$i}\n";
}
print LOG "\nMANCOLL\n";
foreach my $i (sort keys %hMANCOLL) {
    print LOG "$i $hMANCOLL{$i}\n";
}
print LOG "\nCLASS1\n";
foreach my $i (sort keys %hCLASS1) {
    print LOG "$i $hCLASS1{$i}\n";
}
print LOG "\nCLASS2\n";
foreach my $i (sort keys %hCLASS2) {
    print LOG "$i $hCLASS2{$i}\n";
}
print LOG "\nOBJCONT\n";
foreach my $i (sort keys %hOBJCONT) {
    print LOG "$i $hOBJCONT{$i}\n";
}
print LOG "\nDVEST\n";
foreach my $i (sort keys %hDVEST) {
    print LOG "$i $hDVEST{$i}\n";
}


# Note: 1=age, 2=gender, 3=sysbp, 4=resprate, 5=pulse
print KEYINFO "seq=1: Intercept\n";
print KEYINFO "seq=2: Age\n";
print KEYINFO "seq=3: DVTOTAL\n";
print KEYINFO "seq=4: SPLIMIT\n";
print KEYINFO "\n";

my $seqNum = 5;		

# assign sequence number for TREATMNT.
foreach my $i (sort keys %hTREATMNT) {
    $hTREATMNT{$i}=$seqNum;
    $seqNum++;        
}
print KEYINFO "TREATMNT Keys... ". (keys %hTREATMNT) ." keys\n";
foreach my $i (sort keys %hTREATMNT) {
    print KEYINFO "seq=$hTREATMNT{$i}: TREATMNT=$i\n";
}
print KEYINFO "\n";

# assign sequence number for PARUSE.
foreach my $i (sort keys %hPARUSE) {
    $hPARUSE{$i}=$seqNum;
    $seqNum++;        
}
print KEYINFO "PARUSE Keys... ". (keys %hPARUSE) ." keys\n";
foreach my $i (sort keys %hPARUSE) {
    print KEYINFO "seq=$hPARUSE{$i}: PARUSE=$i\n";
}
print KEYINFO "\n";

# assign sequence number for CHORIENTA.
foreach my $i (sort keys %hCHORIENTA) {
    $hCHORIENTA{$i}=$seqNum;
    $seqNum++;        
}
print KEYINFO "CHORIENTA Keys... ". (keys %hCHORIENTA) ." keys\n";
foreach my $i (sort keys %hCHORIENTA) {
    print KEYINFO "seq=$hCHORIENTA{$i}: CHORIENTA=$i\n";
}
print KEYINFO "\n";

# assign sequence number for SEATPOS_ROW
foreach my $i (sort keys %hSEATPOS_ROW){
    $hSEATPOS_ROW{$i}=$seqNum;
    $seqNum++;        
}
print KEYINFO "SEATPOS_ROW Keys... ". (keys %hSEATPOS_ROW) ." keys\n";
foreach my $i (sort keys %hSEATPOS_ROW){
    print KEYINFO "seq=$hSEATPOS_ROW{$i}, SEATPOS_ROW=$i\n";
}
print KEYINFO "\n";

# assign sequence number for SEATPOS_COL 
foreach my $i (sort keys %hSEATPOS_COL){
    $hSEATPOS_COL{$i}=$seqNum;
    $seqNum++;        
}
print KEYINFO "SEATPOS_COL Keys... ". (keys %hSEATPOS_COL) ." keys\n";
foreach my $i (sort keys %hSEATPOS_COL){
    print KEYINFO "seq=$hSEATPOS_COL{$i}, SEATPOS_COL=$i\n";
}
print KEYINFO "\n";

# assign sequence number for EJECTION
foreach my $i (sort keys %hEJECTION){
    $hEJECTION{$i}=$seqNum;
    $seqNum++;        
}
print KEYINFO "EJECTION Keys... ". (keys %hEJECTION) ." keys\n";
foreach my $i (sort keys %hEJECTION){
    print KEYINFO "seq=$hEJECTION{$i}, EJECTION=$i\n";
}
print KEYINFO "\n";

# assign sequence number for GLASGOW
foreach my $i (sort keys %hGLASGOW){
    $hGLASGOW{$i}=$seqNum;
    $seqNum++;        
}
print KEYINFO "GLASGOW Keys... ". (keys %hGLASGOW) ." keys\n";
foreach my $i (sort keys %hGLASGOW){
    print KEYINFO "seq=$hGLASGOW{$i}, GLASGOW=$i\n";
}
print KEYINFO "\n";

# assign sequence number for AISCODE
foreach my $i (sort keys %hAISCODE){
    $hAISCODE{$i}=$seqNum;
    $seqNum++;        
}
print KEYINFO "AISCODE Keys... ". (keys %hAISCODE) ." keys\n";
foreach my $i (sort keys %hAISCODE){
    print KEYINFO "seq=$hAISCODE{$i}: AISCODE=$i\n";
}
print KEYINFO "\n";

# assign sequence number for MANCOLL
foreach my $i (sort keys %hMANCOLL){
    $hMANCOLL{$i}=$seqNum;
    $seqNum++;        
}
print KEYINFO "MANCOLL Keys... ". (keys %hMANCOLL) ." keys\n";
foreach my $i (sort keys %hMANCOLL){
    print KEYINFO "seq=$hMANCOLL{$i}, MANCOLL=$i\n";
}
print KEYINFO "\n";

# assign sequence number for CLASS1
foreach my $i (sort keys %hCLASS1){
    $hCLASS1{$i}=$seqNum;
    $seqNum++;        
}
print KEYINFO "CLASS1 Keys... ". (keys %hCLASS1) ." keys\n";
foreach my $i (sort keys %hCLASS1){
    print KEYINFO "seq=$hCLASS1{$i}, CLASS1=$i\n";
}
print KEYINFO "\n";

# assign sequence number for CLASS2
foreach my $i (sort keys %hCLASS2){
    $hCLASS2{$i}=$seqNum;
    $seqNum++;        
}
print KEYINFO "CLASS2 Keys... ". (keys %hCLASS2) ." keys\n";
foreach my $i (sort keys %hCLASS2){
    print KEYINFO "seq=$hCLASS2{$i}, CLASS2=$i\n";
}
print KEYINFO "\n";

# assign sequence number for OBJCONT
foreach my $i (sort keys %hOBJCONT){
    $hOBJCONT{$i}=$seqNum;
    $seqNum++;        
}
print KEYINFO "OBJCONT Keys... ". (keys %hOBJCONT) ." keys\n";
foreach my $i (sort keys %hOBJCONT){
    print KEYINFO "seq=$hOBJCONT{$i}, OBJCONT=$i\n";
}
print KEYINFO "\n";

# assign sequence number for DVEST
foreach my $i (sort keys %hDVEST){
    $hDVEST{$i}=$seqNum;
    $seqNum++;        
}
print KEYINFO "DVEST Keys... ". (keys %hDVEST) ." keys\n";
foreach my $i (sort keys %hDVEST){
    print KEYINFO "seq=$hDVEST{$i}, DVEST=$i\n";
}
print KEYINFO "\n";

# start to print out data
for(my $i = 1; $i <= $nTrainFiles; $i++) {
    
    my $HD_FILE_NAME = $BASE_HD_FILE_NAME . "_CDS_" . $trainFrac_CDS . "_NTDB_" . $trainFrac_NTDB . "_" . $i . ".txt";
    my $LD_FILE_NAME = $BASE_LD_FILE_NAME . "_CDS_" . $trainFrac_CDS . "_NTDB_" . $trainFrac_NTDB . "_" . $i . ".txt";
    
    my ($nCDS_NULL, $nCDS_Alive, $nCDS_Dead, $nNTDB_TRAIN_Alive, $nNTDB_TRAIN_Dead);
    $nCDS_NULL = floor(0.5 + $trainFrac_CDS*($#aInds_CDS_NULL + 1));
    $nCDS_Alive = floor(0.5 + $trainFrac_CDS*($#aInds_CDS_Alive + 1));
    $nCDS_Dead = floor(0.5 + $trainFrac_CDS*($#aInds_CDS_Dead + 1));
    $nNTDB_TRAIN_Alive = floor(0.5 + $trainFrac_NTDB*($#aInds_NTDB_TRAIN_Alive + 1));
    $nNTDB_TRAIN_Dead = floor(0.5 + $trainFrac_NTDB*($#aInds_NTDB_TRAIN_Dead + 1));

    my @aInds_selected = (shuffle(@aInds_CDS_NULL))[0 .. $nCDS_NULL-1];    
    push(@aInds_selected, (shuffle(@aInds_CDS_Alive))[0 .. $nCDS_Alive-1]);
    push(@aInds_selected, (shuffle(@aInds_CDS_Dead))[0 .. $nCDS_Dead-1]);
    push(@aInds_selected, (shuffle(@aInds_NTDB_TRAIN_Alive))[0 .. $nNTDB_TRAIN_Alive-1]);
    push(@aInds_selected, (shuffle(@aInds_NTDB_TRAIN_Dead))[0 .. $nNTDB_TRAIN_Dead-1]);
    push(@aInds_selected, @aInds_NTDB_TEST_Alive);
    push(@aInds_selected, @aInds_NTDB_TEST_Dead);
    @aInds_selected = sort{$a <=> $b}(@aInds_selected);

#    print join(", ", @aInds_selected);
    
    my @aUID_selected = @aUID[@aInds_selected];
    
    if ($trainFrac_CDS == 1.0 and $trainFrac_NTDB == 1.0) {
        $HD_FILE_NAME = $BASE_HD_FILE_NAME . '.txt';
        $LD_FILE_NAME = $BASE_LD_FILE_NAME . '.txt';
        @aUID_selected = @aUID;
    }
    
    open ($CDS_NTDB_HD, '> ' , $HD_FILE_NAME); # CDS and NTDB concatenated as: CDS + NTDB_Train + NTDB_Test for imputation
    open ($CDS_onlyNTDB_LD, '> ' , $LD_FILE_NAME); # Just NTDB as: NTDB_Train + NTDB_Test for imputation
    
    foreach my $key (@aUID_selected){
        my $index=0;
    
        # DEATH
        if($hDataType{$key} ne 'NTDB_TEST') {
            print $CDS_NTDB_HD $hDEATH{$key} . " ";
            if($hDataType{$key} ne 'CDS') {
                print $CDS_onlyNTDB_LD $hDEATH{$key} . " ";
            }
        } else {
            print $CDS_NTDB_HD "NULL ";
            if($hDataType{$key} ne 'CDS') {
                print $CDS_onlyNTDB_LD "NULL ";
            }
            if ($i == 1) {
                print $CDS_NTDB_OUTPUT_TEST $hDEATH{$key} . "\n";
            }
        }
        
        # Intercept    
        $index++;
        print $CDS_NTDB_HD "1:1 ";
        if($hDataType{$key} ne 'CDS') {
            print $CDS_onlyNTDB_LD "1:1 ";
        }
        
        # AGE    
        $index++;
        print $CDS_NTDB_HD "$index:$hAGE{$key} ";
        if($hDataType{$key} ne 'CDS') {
            print $CDS_onlyNTDB_LD "$index:$hAGE{$key} ";
        }
    
        # DVTOTAL
        $index++;
        print $CDS_NTDB_HD "$index:$hDVTOTAL{$key} ";
        
        # SPLIMIT
        $index++;
        print $CDS_NTDB_HD "$index:$hSPLIMIT{$key} ";
        
        # TREATMNT
        foreach my $j (sort keys %{$hUID_TREATMNT{$key}}){
            if($j ne "NULL"){
                print $CDS_NTDB_HD $hTREATMNT{$j} . ":1 ";
            }
            else{
                foreach my $k (sort keys %hTREATMNT){
                    if($k ne "NULL"){
                        print $CDS_NTDB_HD $hTREATMNT{$k} . ":NULL ";
                    }
                }
            }
        }
        
        # PARUSE
        foreach my $j (sort keys %{$hUID_PARUSE{$key}}){
            if($j ne "NULL"){
                if($j == 5) {
                    foreach my $k (sort keys %hPARUSE){
                        if($k ne "NULL" and $k != 1 and $k != 5){
                            print $CDS_NTDB_HD $hPARUSE{$k} . ":NULL ";
                        }
                    }   
                } else {
                    print $CDS_NTDB_HD $hPARUSE{$j} . ":1 ";
                }
            }
            else{
                foreach my $k (sort keys %hPARUSE){
                    if($k ne "NULL" and $k != 5){
                        print $CDS_NTDB_HD $hPARUSE{$k} . ":NULL ";
                    }
                }
            }
        }
         
        #CHORIENTA
        foreach my $j (sort keys %{$hUID_CHORIENTA{$key}}){
            if($j ne "NULL"){
                print $CDS_NTDB_HD $hCHORIENTA{$j} . ":1 ";
            }
            else{
                foreach my $k (sort keys %hCHORIENTA){
                    if($k ne "NULL"){
                        print $CDS_NTDB_HD $hCHORIENTA{$k} . ":NULL ";
                    }
                }
            }
        }
        
        #SEATPOS_ROW
        foreach my $j (sort keys %{$hUID_SEATPOS_ROW{$key}}){
            if($j ne "NULL"){
                print $CDS_NTDB_HD $hSEATPOS_ROW{$j} . ":1 ";
            }
            else{
                foreach my $k (sort keys %hSEATPOS_ROW){
                    if($k ne "NULL"){
                        print $CDS_NTDB_HD $hSEATPOS_ROW{$k} . ":NULL ";
                    }
                }
            }
        }
        
        #SEATPOS_COL
        foreach my $j (sort keys %{$hUID_SEATPOS_COL{$key}}){
            if($j ne "NULL"){
                print $CDS_NTDB_HD $hSEATPOS_COL{$j} . ":1 ";
            }
            else{
                foreach my $k (sort keys %hSEATPOS_COL){
                    if($k ne "NULL"){
                        print $CDS_NTDB_HD $hSEATPOS_COL{$k} . ":NULL ";
                    }
                }
            }
        }
        
        #EJECTION
        foreach my $j (sort keys %{$hUID_EJECTION{$key}}){
            if($j ne "NULL"){
                print $CDS_NTDB_HD $hEJECTION{$j} . ":1 ";
            }
            else{
                foreach my $k (sort keys %hEJECTION){
                    if($k ne "NULL"){
                        print $CDS_NTDB_HD $hEJECTION{$k} . ":NULL ";
                    }
                }
            }
        }
        
        #GLASGOW
        foreach my $j (sort keys %{$hUID_GLASGOW{$key}}){
            if($j ne "NULL"){
                print $CDS_NTDB_HD $hGLASGOW{$j} . ":1 ";
                if($hDataType{$key} ne 'CDS') {
                    print $CDS_onlyNTDB_LD $hGLASGOW{$j} . ":1 ";
                }
            }
            else{
                foreach my $k (sort keys %hGLASGOW){
                    if($k ne "NULL"){
                        print $CDS_NTDB_HD $hGLASGOW{$k} . ":NULL ";
                        if($hDataType{$key} ne 'CDS') {
                            print $CDS_onlyNTDB_LD $hGLASGOW{$k} . ":NULL ";
                        }
                    }
                }
            }
        }
        
        #AISCODE
        foreach my $j (sort keys %{$hUID_AISCODE{$key}}){
            if($j ne "NULL"){
                print $CDS_NTDB_HD $hAISCODE{$j} . ":1 ";
                if($hDataType{$key} ne 'CDS') {
                    print $CDS_onlyNTDB_LD $hAISCODE{$j} . ":1 ";
                }
            }
        }
        
        #MANCOLL
        foreach my $j (sort keys %{$hUID_MANCOLL{$key}}){
            if($j ne "NULL"){
                print $CDS_NTDB_HD $hMANCOLL{$j} . ":1 ";
            }
            else{
                foreach my $k (sort keys %hMANCOLL){
                    if($k ne "NULL"){
                        print $CDS_NTDB_HD $hMANCOLL{$k} . ":NULL ";
                    }
                }
            }
        }
        
        #CLASS1
        foreach my $j (sort keys %{$hUID_CLASS1{$key}}){
            if($j ne "NULL"){
                print $CDS_NTDB_HD $hCLASS1{$j} . ":1 ";
            }
            else{
                foreach my $k (sort keys %hCLASS1){
                    if($k ne "NULL"){
                        print $CDS_NTDB_HD $hCLASS1{$k} . ":NULL ";
                    }
                }
            }
        }
        
        #CLASS2
        foreach my $j (sort keys %{$hUID_CLASS2{$key}}){
            if($j ne "NULL"){
                print $CDS_NTDB_HD $hCLASS2{$j} . ":1 ";
            }
            else{
                foreach my $k (sort keys %hCLASS2){
                    if($k ne "NULL"){
                        print $CDS_NTDB_HD $hCLASS2{$k} . ":NULL ";
                    }
                }
            }
        }
        
        #OBJCONT
        foreach my $j (sort keys %{$hUID_OBJCONT{$key}}){
            if($j ne "NULL"){
                print $CDS_NTDB_HD $hOBJCONT{$j} . ":1 ";
            }
            else{
                foreach my $k (sort keys %hOBJCONT){
                    if($k ne "NULL"){
                        print $CDS_NTDB_HD $hOBJCONT{$k} . ":NULL ";
                    }
                }
            }
        }
        
        #DVEST
        foreach my $j (sort keys %{$hUID_DVEST{$key}}){
            if($j ne "NULL"){
                print $CDS_NTDB_HD $hDVEST{$j} . ":1 ";
            }
            else{
                foreach my $k (sort keys %hDVEST){
                    if($k ne "NULL"){
                        print $CDS_NTDB_HD $hDVEST{$k} . ":NULL ";
                    }
                }
            }
        }
    
        print $CDS_NTDB_HD "\n";
        if($hDataType{$key} ne 'CDS') {
            print $CDS_onlyNTDB_LD "\n";
        }
    }
    close($CDS_NTDB_HD);
    close($CDS_onlyNTDB_LD);
}

$cds->disconnect();
close(KEYINFO);
close(LOG);
close($CDS_NTDB_OUTPUT_TEST);
